{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tweepy\n",
    "import twitter\n",
    "\n",
    "import csv\n",
    "from datetime import datetime \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "\n",
    "CONSUMER_KEY = 'nHSSMHieBZ4glVM9U7QGtI81r'\n",
    "CONSUMER_SECRET = 'Wsm8z4oxzpq95aajc3El8TMlbmWTD3Vo8EvZ8a6vJta922KJsS'\n",
    "OAUTH_TOKEN = '721095779335548929-xHLTocT6ClXB2LBkFMZbBRAsLBCjtWt'\n",
    "OAUTH_TOKEN_SECRET = 'JwYvCraZvEMP1JlopqbRLvOZJI1VfVMQFvEoXAAQ4wksO'\n",
    "\n",
    "auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET,\n",
    "                           CONSUMER_KEY, CONSUMER_SECRET)\n",
    "\n",
    "twitter_api = twitter.Twitter(auth=auth)\n",
    "\n",
    "authT = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)\n",
    "authT.set_access_token(OAUTH_TOKEN, OAUTH_TOKEN_SECRET)\n",
    "api = tweepy.API(authT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "WORLD_WOE_ID = 1\n",
    "US_WOE_ID = 23424977\n",
    "\n",
    "# Prefix ID with the underscore for query string parameterization.\n",
    "# Without the underscore, the twitter package appends the ID value\n",
    "# to the URL itself as a special case keyword argument.\n",
    "\n",
    "world_trends = twitter_api.trends.place(_id=WORLD_WOE_ID)\n",
    "us_trends = twitter_api.trends.place(_id=US_WOE_ID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "print (json.dumps(US_WOE_ID, indent=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "import bs4\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "us_trends_set = list([trend['name'] \n",
    "                        for trend in us_trends[0]['trends']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveFile(df):\n",
    "    csv = '/Users/LXIN/Desktop/twitter.csv'\n",
    "    \n",
    "    if not os.path.isfile(csv):\n",
    "        df.to_csv(csv, header=True, index=False, encoding='utf-8')\n",
    "        print(\"new\")\n",
    "    else:\n",
    "        df.to_csv(csv, mode='a', header=False, index=False)\n",
    "        print(\"exist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveFrame(data, topics, time, data_len):\n",
    "    colName = ['Date', 'Topic', 'Comment']\n",
    "    df = pd.DataFrame(columns = colName)\n",
    "    timeLst = []\n",
    "    topicLst = []\n",
    "    \n",
    "    for i in range(data_len):\n",
    "        timeLst.append(time)\n",
    "        topicLst.append(topics)\n",
    "    \n",
    "    df['Date'] = timeLst\n",
    "    df['Topic'] = topicLst\n",
    "    df['Comment'] = data\n",
    "    \n",
    "    saveFile(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import emoji\n",
    "\n",
    "# TEST Remove all the emoji \n",
    "test = \"eak-up juicy details 😩😩\"\n",
    "\n",
    "text = ''.join(c for c in test if c not in emoji.UNICODE_EMOJI)\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# XXX: Set this variable to a trending topic, \n",
    "# or anything else for that matter. The example query below\n",
    "# was a trending topic when this content was being developed\n",
    "# and is used throughout the remainder of this chapter.\n",
    "\n",
    "import emoji\n",
    "import time\n",
    "\n",
    "now = datetime.now().strftime(\"%m/%d/%Y, %H:%M:%S\")\n",
    "\n",
    "#print(now)\n",
    "\n",
    "\n",
    "# Remove the \"#\" in front of the topic\n",
    "for i in range(0,30):\n",
    "    q = us_trends_set[i]\n",
    "    \n",
    "    topics = []\n",
    "    topics.append(q.replace(\"#\",\"\"))\n",
    "    time = []\n",
    "    time.append(now)\n",
    "    data = []\n",
    "    \n",
    "\n",
    "    # Total number of tweets extracted: 30\n",
    "    # Extract the most popular tweets under each trending topic\n",
    "    tweets = api.search(q=q, result_type='popular', count=50, include_rts = False, tweet_mode = 'extended')\n",
    "    \n",
    "    for info in tweets:\n",
    "        \n",
    "        text = ''.join(c for c in info.full_text if c not in emoji.UNICODE_EMOJI)    \n",
    "        data.append(text)\n",
    "\n",
    "    # If the number of popular tweets is less than 30, extract some latest tweets \n",
    "    tweets2 = api.search(q=q , count= (30 - len(data)), include_rts = False, tweet_mode = 'extended')\n",
    "    for info in tweets2:\n",
    "        \n",
    "        text = ''.join(c for c in info.full_text if c not in emoji.UNICODE_EMOJI)  \n",
    "        \n",
    "        data.append(text)\n",
    "\n",
    "    # Store the datetime, trending topic and tweets to file\n",
    "    saveFrame(data, topics, time, len(data))\n",
    "\n",
    "\n",
    "print(\"done\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "# data = pd.read_csv(\"/Users/LXIN/Desktop/twitter.csv\")\n",
    "# print(data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
